import re

import russian_stemmer
import english_stemmer

stemmers = {
    'english' : english_stemmer.stem,
    'russian' : russian_stemmer.stem
    }

ASCII_SIZE = 128

class Splitter:
    tagRegexp = re.compile(r'<[^>]*>', re.U)
    escapeRegexp = re.compile(r'&#?[\w]*?;', re.U)
    commentRegexp = re.compile(r'<!-.*?-->', re.U)
    wordRegexp = re.compile('\w+', re.U)

    def stripHtml(self, html):
        """
        Removes HTML tags, escape sequences and comments
        """
        tagStrippedHtml = self.tagRegexp.sub('', html)
        escapeStrippedHtml = self.escapeRegexp.sub('', tagStrippedHtml)
        commentStrippedHtml = self.commentRegexp.sub('', escapeStrippedHtml)
        return commentStrippedHtml

    def splitIntoWords(self, text):
        return self.wordRegexp.findall(text)

_stripper = None

def getTextBases(text):
    """
    Strips the text of all HTML and returns the word bases found in it.
    """
    global _stripper
    if _stripper is None:
        _stripper = Splitter()
    text = _stripper.stripHtml(text)
    words = _stripper.splitIntoWords(text)
    return map(getWordBase, words)

def getWordBase(word):
    """
    Returns the base of the word depending on its language.
    """
    stemmer = getStemmer(word)
    result = stemmer(word)
    return result

def getStemmer(word):
    if ord(word[0]) < ASCII_SIZE:
        return stemmers['english']
    else:
        return stemmers['russian']
